{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\54554\\\\Desktop\\\\Project\\\\race-nlp'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['README.md', 'test', 'train', 'valid']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"dict_to_csv\"\"\"\n",
    "#实例\n",
    "# def TextProcessing(folder_path: object, test_size: object = 0.2) -> object:\n",
    "#     \"\"\"\n",
    "#     划分数据集和测试集合\n",
    "#     \"\"\"\n",
    "#     folder_list = os.listdir(folder_path)\n",
    "#     data_list = []\n",
    "#     class_list = []\n",
    "\n",
    "#     # 类间循环\n",
    "#     for folder in folder_list:\n",
    "#         new_folder_path = os.path.join(folder_path, folder)\n",
    "#         print(new_folder_path)\n",
    "#         files = os.listdir(new_folder_path)\n",
    "#         print(files)\n",
    "#         # 类内循环\n",
    "#         j = 0\n",
    "#         for file in files:\n",
    "#             if j > 410: # 每类text样本数最多410\n",
    "#                 break\n",
    "#             with open(os.path.join(new_folder_path, file), 'r', encoding='UTF-8') as fp:\n",
    "#                raw = fp.read()\n",
    "#             ## ---------------------分词-------------------------------------------------------\n",
    "#             word_cut = jieba.cut(raw, cut_all=False) # 精确模式，返回的结构是一个可迭代的genertor\n",
    "\n",
    "\n",
    "\n",
    "#             word_list = list(word_cut) # genertor转化为list，每个词unicode格式\n",
    "#             ## --------------------------------------------------------------------------------\n",
    "#             data_list.append(word_list)\n",
    "#             class_list.append(folder)\n",
    "#             j += 1\n",
    "folder_list_1 = os.listdir(\"./data/CLOTH/\")\n",
    "folder_list_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"lab\"\"\"\n",
    "from pathlib import Path\n",
    "path1 = Path(\"./data/\")\n",
    "#path1\n",
    "path_dev = path1/'dev'\n",
    "path_train = path1/'train'\n",
    "path_test = path1/'test'\n",
    "\n",
    "level = []\n",
    "\n",
    "files = os.listdir(path_train)\n",
    "for file in files:\n",
    "    new_files_path = os.path.join(path_train, file)\n",
    "    #print(new_files_path)\n",
    "    txt_files = os.listdir(new_files_path)\n",
    "    #print(os.listdir(new_files_path))\n",
    "#    for txt in txt_files:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "yes\n"
     ]
    }
   ],
   "source": [
    "for file in txt_files:\n",
    "    if file == \"8.txt\":\n",
    "        print(\"yes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25137\n"
     ]
    }
   ],
   "source": [
    "\"\"\"加载文件\"\"\"\n",
    "import json\n",
    "import logging\n",
    "def load_data(in_file, max_example=None, relabeling=True, question_belong=[]):\n",
    "    documents = []\n",
    "#     questions = []\n",
    "#     answers = []\n",
    "#     options = []\n",
    "    level = []\n",
    "    num_examples = 0\n",
    "    def get_file(path,level = []):\n",
    "        \"\"\"得到子文件路径，说明被调用了3次\"\"\"\n",
    "        files = []\n",
    "        #print(os.listdir(path))\n",
    "        for inf in os.listdir(path):\n",
    "            new_path = os.path.join(path, inf)#更新子路径\n",
    "\n",
    "            if os.path.isdir(new_path):\n",
    "                assert inf in [\"middle\", \"high\"]#断言是否包含middle和high两类\n",
    "                files += get_file(new_path)\n",
    "            else:\n",
    "                files += [new_path]#将目标文档路径汇总\n",
    "                #print(new_path)\n",
    "        return files\n",
    "    files = get_file(in_file)\n",
    "    #print(files)\n",
    "    for inf in files:\n",
    "        obj = json.load(open(inf, \"r\"))\n",
    "        documents += [obj[\"article\"]]\n",
    "        if 'high' in str(inf):\n",
    "            level +=[\"high\"]\n",
    "        else:\n",
    "            level +=[\"middle\"]\n",
    "#         for i, q in enumerate(obj[\"questions\"]):\n",
    "#             question_belong += [inf + \"_\" + str(i)]\n",
    "#             documents += [obj[\"article\"]]\n",
    "#             questions += [q]\n",
    "#             assert len(obj[\"options\"][i]) == 4\n",
    "#             options += obj[\"options\"][i]\n",
    "#             answers += [ord(obj[\"answers\"][i]) - ord('A')]\n",
    "        num_examples += 1\n",
    "        if (max_example is not None) and (num_examples >= max_example):\n",
    "            break\n",
    "    def clean(st_list):\n",
    "        \"\"\"统一为小写字母\"\"\"\n",
    "        for i, st in enumerate(st_list):\n",
    "            st_list[i] = st.lower().strip()\n",
    "        return st_list\n",
    "    documents = clean(documents)\n",
    "    level = clean(level)\n",
    "#     questions = clean(questions)\n",
    "#     options = clean(options)\n",
    "    print(len(documents))\n",
    "    #logging.info('#Examples: %d' % len(documents))\n",
    "    return (documents,level)\n",
    "\n",
    "#doc文章\n",
    "#lev类别"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"得到train.csv\"\"\"\n",
    "doc,lev=load_data('./data/train/',)\n",
    "data_class_list = list(zip(doc, lev))\n",
    "#data_class_list?\n",
    "name=[\"article\",\"class\"]\n",
    "data_csv = pd.DataFrame(columns=name,data=data_class_list)\n",
    "#data_csv[0:2]\n",
    "data_csv.to_csv('./data/train.csv',encoding=\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>studies show that you may be lied to every day...</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>you could feel sorry for alberto torres , who ...</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>although most weddings follow long-held tradit...</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>it was 1961 and i was in the fifth grade . my ...</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>the sat is one of two major tests for the entr...</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1402</th>\n",
       "      <td>come and see the india elephants and the new t...</td>\n",
       "      <td>middle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1403</th>\n",
       "      <td>what are your clothes made of ? are they made ...</td>\n",
       "      <td>middle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1404</th>\n",
       "      <td>most people do n't like mice , but they love o...</td>\n",
       "      <td>middle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1405</th>\n",
       "      <td>val is a six-year-old boy . he begins to go to...</td>\n",
       "      <td>middle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1406</th>\n",
       "      <td>my name is david and i have two brothers , mar...</td>\n",
       "      <td>middle</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1407 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                article   class\n",
       "0     studies show that you may be lied to every day...    high\n",
       "1     you could feel sorry for alberto torres , who ...    high\n",
       "2     although most weddings follow long-held tradit...    high\n",
       "3     it was 1961 and i was in the fifth grade . my ...    high\n",
       "4     the sat is one of two major tests for the entr...    high\n",
       "...                                                 ...     ...\n",
       "1402  come and see the india elephants and the new t...  middle\n",
       "1403  what are your clothes made of ? are they made ...  middle\n",
       "1404  most people do n't like mice , but they love o...  middle\n",
       "1405  val is a six-year-old boy . he begins to go to...  middle\n",
       "1406  my name is david and i have two brothers , mar...  middle\n",
       "\n",
       "[1407 rows x 2 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1407\n"
     ]
    }
   ],
   "source": [
    "\"\"\"得到train.csv\"\"\"\n",
    "doc,lev=load_data('./data/test/',)\n",
    "data_class_list = list(zip(doc, lev))\n",
    "#data_class_list?\n",
    "name=[\"article\",\"class\"]\n",
    "data_csv = pd.DataFrame(columns=name,data=data_class_list)\n",
    "#data_csv[0:2]\n",
    "data_csv.to_csv('./data/test.csv',encoding=\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"my husband is a born shopper . he loves to look at things and to touch them . he likes to compare prices between the same items in different shops . he would never think of buying anything without looking around in several different shops . on the other hand , i 'm not a shopper . i think shopping is boring and unpleasant . if i like something and i have enough money to take it , i buy it at once . i never look around for a good price or a better deal . of course my husband and i never go shopping together . doing shopping together would be too painful for both of us . when it comes to shopping , we go our different ways . sometimes i ask my son jimmy to buy some food in the shop not far from our home . but he is always absent-minded . this was his story . one day i said to him , `` i hope you wo n't forget what i have told you to buy . '' `` no , '' said jimmy . `` i wo n't forget . you want three oranges , six eggs and a pound of meat . '' he went running down the street to the shop . as he ran , he said to himself over and over again , `` three oranges , six eggs and a pound of meat . '' in the beginning he remembered everything but he stopped several times . once he saw two men fighting outside a clothes shop until a policeman stopped them . one of them was badly hurt . then he stopped to give ten cents to a beggar . then he met some of his friends and he played with them for a while . when he reached the shop , he had forgotten everything except six eggs . as he walked home , his face became sadder and sadder . when he saw me he said , `` i 'm sorry , mum . i have forgotten to buy oranges and the meat . i only remembered to buy six eggs , but i 've dropped three of them . ''\",\n",
       " \"my husband is a born shopper . he loves to look at things and to touch them . he likes to compare prices between the same items in different shops . he would never think of buying anything without looking around in several different shops . on the other hand , i 'm not a shopper . i think shopping is boring and unpleasant . if i like something and i have enough money to take it , i buy it at once . i never look around for a good price or a better deal . of course my husband and i never go shopping together . doing shopping together would be too painful for both of us . when it comes to shopping , we go our different ways . sometimes i ask my son jimmy to buy some food in the shop not far from our home . but he is always absent-minded . this was his story . one day i said to him , `` i hope you wo n't forget what i have told you to buy . '' `` no , '' said jimmy . `` i wo n't forget . you want three oranges , six eggs and a pound of meat . '' he went running down the street to the shop . as he ran , he said to himself over and over again , `` three oranges , six eggs and a pound of meat . '' in the beginning he remembered everything but he stopped several times . once he saw two men fighting outside a clothes shop until a policeman stopped them . one of them was badly hurt . then he stopped to give ten cents to a beggar . then he met some of his friends and he played with them for a while . when he reached the shop , he had forgotten everything except six eggs . as he walked home , his face became sadder and sadder . when he saw me he said , `` i 'm sorry , mum . i have forgotten to buy oranges and the meat . i only remembered to buy six eggs , but i 've dropped three of them . ''\",\n",
       " \"my husband is a born shopper . he loves to look at things and to touch them . he likes to compare prices between the same items in different shops . he would never think of buying anything without looking around in several different shops . on the other hand , i 'm not a shopper . i think shopping is boring and unpleasant . if i like something and i have enough money to take it , i buy it at once . i never look around for a good price or a better deal . of course my husband and i never go shopping together . doing shopping together would be too painful for both of us . when it comes to shopping , we go our different ways . sometimes i ask my son jimmy to buy some food in the shop not far from our home . but he is always absent-minded . this was his story . one day i said to him , `` i hope you wo n't forget what i have told you to buy . '' `` no , '' said jimmy . `` i wo n't forget . you want three oranges , six eggs and a pound of meat . '' he went running down the street to the shop . as he ran , he said to himself over and over again , `` three oranges , six eggs and a pound of meat . '' in the beginning he remembered everything but he stopped several times . once he saw two men fighting outside a clothes shop until a policeman stopped them . one of them was badly hurt . then he stopped to give ten cents to a beggar . then he met some of his friends and he played with them for a while . when he reached the shop , he had forgotten everything except six eggs . as he walked home , his face became sadder and sadder . when he saw me he said , `` i 'm sorry , mum . i have forgotten to buy oranges and the meat . i only remembered to buy six eggs , but i 've dropped three of them . ''\"]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Object `obj` not found.\n"
     ]
    }
   ],
   "source": [
    "obj?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[1;31mType:\u001b[0m        list\n",
       "\u001b[1;31mString form:\u001b[0m [2, 0, 1, 2, 1, 0, 2, 3, 2, 2, 2, 3, 2, 1, 2, 1, 2, 1, 1, 3, 0, 1, 1, 2, 0, 2, 0, 3, 2, 3, 3, 1,  <...> , 1, 2, 2, 3, 0, 3, 0, 3, 0, 3, 2, 3, 1, 2, 3, 1, 2, 3, 1, 0, 1, 3, 3, 0, 2, 2, 3, 0, 1, 1, 3, 0]\n",
       "\u001b[1;31mLength:\u001b[0m      101\n",
       "\u001b[1;31mDocstring:\u001b[0m  \n",
       "Built-in mutable sequence.\n",
       "\n",
       "If no argument is given, the constructor creates a new empty list.\n",
       "The argument must be an iterable if specified.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ans?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py_torch",
   "language": "python",
   "name": "py_torch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
